/**
 *	Ellipse, treated as one token
 *	Multiple Question and Exclamation Marks, treated as one token
 */

/**
 *	@author: Manimin R. Morga
 *	date created: January 8, 2005
 *	version: 1.08
 *	modified: May 18, 2006
 */

package text.ia;

import java.util.Iterator;
import java.util.Vector;
import java.util.regex.Pattern;

public class TExtTokenizer implements TokenizerInterface{
	
	private String strSentence;
	private Vector vTokens;
	private TExtLexicon lex;

	public TExtTokenizer() {
		strSentence = null;
		vTokens = null;
		lex = null;
	}

	protected StringBuffer checkTokenIfContraction(StringBuffer strbToken) {
		Vector vContraction = null;
		int j;
		
		//token
		if(strbToken != null && strbToken.length() > 0) {
			//check if token is a contraction
			//System.out.println("candidate contraction: "+strbToken);
			if((vContraction = determineContraction(strbToken.toString().toLowerCase())) != null) {
				//System.out.println("contraction");
				//for first char uppercase contraction
				if(Character.isUpperCase(strbToken.charAt(0))) {
					strSentence = strSentence.replaceFirst(strbToken.toString(), ((String)vContraction.get(0)).replace(Character.toLowerCase(strbToken.charAt(0)), strbToken.charAt(0)));
				}else {
					strSentence = strSentence.replaceFirst(strbToken.toString(), (String)vContraction.get(0));
				}
				
				String[] strArrContraction = ((String)vContraction.get(0)).split(" ");
				for(j = 0; j < strArrContraction.length; j++) {
					//for uppercase contraction
					if(j == 0 && Character.isUpperCase(strbToken.charAt(0))) {
						vTokens.add(strArrContraction[j].replace(Character.toLowerCase(strbToken.charAt(0)), strbToken.charAt(0)));
						//System.out.println("uncontract: "+strArrContraction[j].replace(Character.toLowerCase(strbToken.charAt(0)), strbToken.charAt(0)));
					}else {
						vTokens.add(strArrContraction[j]);
						//System.out.println(strArrContraction[j]);
					}
				}
				vContraction = null;
				strbToken = new StringBuffer();
			}else {
				vTokens.add(strbToken.toString());
				strbToken = new StringBuffer();
			}
		}
		
		return strbToken;
	}

	protected boolean checkTokenIfNumber(int nIndex, StringBuffer strbToken, String strOrigSentence) {
		boolean bPeriodIsForNumber = false;
		//System.out.println("\tcandidate: "+strbToken);
		if ((isNumber(strbToken.toString()) && 
				nIndex < strOrigSentence.length() - 1 && 
				Character.isDigit(strOrigSentence.charAt(nIndex + 1))) ||
				(nIndex < strOrigSentence.length() - 1 && 
				Character.isDigit(strOrigSentence.charAt(nIndex + 1)))) {
			bPeriodIsForNumber = true;
		}

		return bPeriodIsForNumber;
	}

	protected int checkTokenIfAbbrev(char cTarget, int nIndex, StringBuffer strbOrigToken, String strOrigSentence) {
		StringBuffer strbToken = new StringBuffer(strbOrigToken.toString());
		int nNewIndex = nIndex-1, nOrigNewIndex, nOrigTokenIndex, nNewTokenIndex;
		boolean bFound = false;
		//get the end of the word from the index including the period
		for(int i = nIndex; i < strOrigSentence.length() && 
							!isDelimiter(strOrigSentence.charAt(i)) && 
							strOrigSentence.charAt(i) != '?' && 
							strOrigSentence.charAt(i) != '!'; i++) {
			//handling of ellipse
			strbToken.append(strOrigSentence.charAt(i));
			nNewIndex++;
		}
		
		//System.out.println("\tcandidate: "+strbToken);

		if (isAbbrev(cTarget, strbToken.toString())) {
			//System.out.println("\tabbrev");
			vTokens.add(strbToken.toString());
		}
		//the abbrev can be followed by an ellipse, therefore is not recognized earlier
		else {
			//The molecule has  at.wt....at.wt.... which is 9.
			//This is the moment Mr....Mr.
			nOrigNewIndex = nNewIndex;
			nOrigTokenIndex = strbToken.length()-1;
			while((nNewTokenIndex = strbToken.lastIndexOf(".")) > -1 && !bFound) {
				//System.out.println("\t\tinside while");
				strbToken = new StringBuffer(strbToken.substring(0, nNewTokenIndex));
				//System.out.println("\t\t2candidate: "+strbToken.toString()+".");
				nNewIndex = nOrigNewIndex - (nOrigTokenIndex - nNewTokenIndex);
				//System.out.println("\t\t2nOrigNewIndex: "+nOrigNewIndex);
				//System.out.println("\t\t2nOrigTokenIndex: "+nOrigTokenIndex);
				//System.out.println("\t\t2nNewTokenIndex: "+nNewTokenIndex);
				//System.out.println("\t\t2nNewIndex: "+nNewIndex);
				if (isAbbrev(cTarget, strbToken.toString()+".")) {
					//System.out.println("\t\t2abbrev: "+strbToken.toString()+".");
					bFound = true;
					//check if the char after the abbrev is an ellipse
					if ((nNewTokenIndex = checkTokenIfEllipse(nNewIndex+1, new StringBuffer(strbToken.toString()+"."), strOrigSentence)) > -1) {
						nNewIndex = nNewTokenIndex;
						//System.out.println("\t\t\t3nNewIndex: "+nNewIndex);
					}
				}
			}
			
			if(!bFound || strbToken.length() <= 0) {
				//if not abbrev the nNewIndex is negative
				nNewIndex = -1;
			}
		}

		//the period in the abbrev can be an eos
		if(nNewIndex >= strOrigSentence.length()-1) {
			//if it is then another period is appended to separate the period in the abbrev
			//and the eos
			//System.out.println("\tadd another period");
			//do this in the sentence also
			strSentence += ".";
			vTokens.add(".");
		}

		//System.out.println("\tnNewIndex: "+nNewIndex);
		return nNewIndex;
	}
		
	protected int checkTokenIfEllipse(int nIndex, StringBuffer strbToken, String strOrigSentence) {
		StringBuffer strbEllipse = new StringBuffer();
		int nPeriods = 0;
		int nNewIndex = nIndex-1;

		for(int i = nIndex; i < strOrigSentence.length() && 
							(Character.isWhitespace(strOrigSentence.charAt(i)) ||
							strOrigSentence.charAt(i) == '.'); i++) {
			if(strOrigSentence.charAt(i) == '.') {
				nPeriods++;
				strbEllipse.append(strOrigSentence.charAt(i));
			}
			nNewIndex++;
		}
		
		if (nPeriods >= 3) {
			//System.out.println("nPeriods: "+nPeriods);
			//System.out.println("ellipse: "+strbEllipse);
			checkTokenIfContraction(strbToken);
			vTokens.add(strbEllipse.toString());
		}else{
			nNewIndex = -1;
		}

		return nNewIndex;
	}

	/**
	 *	Accepts a String sentence and returns 
	 *	a Vector containing Strings of tokens.
	 *
	 *	@return Vector of tokens
	 *	@param String strSentence :
	 *		the sentence to be tokenized
	 *	@param char cTargetLanguage :
	 *		the target language (e.g. 'f' for 
	 *		Filipino and , 'e' for English)
	 */
	 /*
	  *	Issues:
	  *		- contraction -> sentence & token
	  *		- ellipse [start, mid, end] [...] - OK
	  *			--> number - OK
	  *			--> abbrev - OK
	  *		- number - OK
	  *		- abbrev - sentence & token: OK
	  *		- n-punctuation - OK
	  */

	public Vector tokenize(String strOrigSentence, char cTarget) {
		strSentence = strOrigSentence;
		vTokens = new Vector();
		lex = new TExtLexicon(cTarget);
		StringBuffer strbToken = new StringBuffer();
		StringBuffer strbCandidateQorEMark;
		int nNewIndex = 0;

		//System.out.println("sentence: "+strSentence);

		for (int i = 0; i < strOrigSentence.length(); i++) {
			if (isDelimiter(strOrigSentence.charAt(i))) {
				//make the previously concatenated chars into a new token
				strbToken = checkTokenIfContraction(strbToken);
				//System.out.println("delimiter: '"+strOrigSentence.charAt(i)+"'");
				//get the delimiter and store it as a new token except if it is a white space
				if (!Character.isWhitespace(strOrigSentence.charAt(i))) {
					//delimiter
					vTokens.add(new String(strOrigSentence.charAt(i)+""));
				}
			}
			//period: ellipse, abbrev, number with decimal pt, end of sentence (last index)
			//[start, middle, end]
			else if(strOrigSentence.charAt(i) == '.') {
				//System.out.println("period");
				//number
				if(checkTokenIfNumber(i, strbToken, strOrigSentence)) {
					//accomodate .232123, 0.1
					//System.out.println("\tnumber");
					strbToken.append(strOrigSentence.charAt(i));
				}
				//abbrev
				else if((nNewIndex = checkTokenIfAbbrev(cTarget, i, strbToken, strOrigSentence)) > -1) {
					//System.out.println("\tinside abbrev");
					i = nNewIndex;
					strbToken = new StringBuffer();
				}
				//eos
				else if(i == strOrigSentence.length() - 1) {
					//make the previously concatenated chars into a new token
					//System.out.println("\ttoken b4 eos: "+strbToken);
					strbToken = checkTokenIfContraction(strbToken);
					//System.out.println("\teos");
					//period
					vTokens.add(new String(strOrigSentence.charAt(i)+""));
				}
				//ellipse
				else if((nNewIndex = checkTokenIfEllipse(i, strbToken, strOrigSentence)) > -1) {
					//number ellipse - OK
					//ellipse number - OK
					//number ellipse number - OK
					//abbrev ellipse - OK
					//ellipse abbrev - OK
					//System.out.println("\tinside ellipse");
					i = nNewIndex;
					strbToken = new StringBuffer();
				}
				//unknown period
				else {
					//System.out.println("\tunknown period");
					checkTokenIfContraction(strbToken);
					vTokens.add(strOrigSentence.charAt(i)+"");
				}
			}
			//n-?|!
			else if ((strOrigSentence.charAt(i) == '?') || (strOrigSentence.charAt(i) == '!')) {
				//make the previously concatenated chars into a new token
				strbToken = checkTokenIfContraction(strbToken);
				//accomodate spaces
				//System.out.println("?|!");

				//if the symbol is not yet the end of sentence char
				//then get all chars from the current index to the last index and group them as a token
				if(i < strOrigSentence.length() - 1) {
					strbCandidateQorEMark = new StringBuffer();

					for(int j = i; j < strOrigSentence.length(); j++) {
						if((strOrigSentence.charAt(j) == '?') || (strOrigSentence.charAt(j) == '!')) {
							strbCandidateQorEMark.append(strOrigSentence.charAt(j));
						}
					}
					
					//ensure that the spaces will not be included
					strSentence = strSentence.replace(strSentence.substring(i,strSentence.length()), strbCandidateQorEMark.toString());
					i = strOrigSentence.length()-1;
					
					vTokens.add(strbCandidateQorEMark.toString());
				}else {
					vTokens.add(new String(strOrigSentence.charAt(i)+""));
				}
			}else {
				//System.out.println("ordinary char: "+strOrigSentence.charAt(i));
				strbToken.append(strOrigSentence.charAt(i));
			}
		}//for-loop

		//for the remaining text not yet added as a token
		checkTokenIfContraction(strbToken);

		vTokens.trimToSize();
		
		return vTokens;
	}

	/**
	 *	Determines if the token being passed is an 
	 *	abbreviation depending on the target sentence.
	 *
	 *	@return boolean True if it is, false otherwise. 
	 *	@param char cTarget The target language of the token (e.g. 'f' for 
	 *		Filipino and , 'e' for English)
	 *	@param String strToken Token
	 */
	protected boolean isAbbrev(char cTarget, String strToken) {
		boolean isAbbrev = lex.lookupAbbrev(strToken.replace("'", "''"));
			
		return isAbbrev;
	}

	/**
	 *	Verify if its a delimiter. A delimiter is : ',', ';', ':', '(', ')',
	 *		'{', '}', '[', ']', '"', '<', '>', '/' and any white space
	 *		('\n', ' ', '\t')
	 *
	 *	@return boolean True if it is, false otherwise. 
	 *	@param char cToken Token
	 */
	protected boolean isDelimiter(char cToken){
		boolean isDelimiter = false;

		if(cToken == ',' || cToken == ';' || cToken == '(' || cToken == ')' 
			|| cToken == '{' || cToken == '}' || cToken == '[' || cToken == ']' 
			|| cToken == '"' || cToken == '<' || cToken == '>' || cToken == '/' 
			|| Character.isWhitespace(cToken))
		   	isDelimiter = true;

		return isDelimiter;
	}

	/**
	 * Verify if its an ellipse.
	 * 
	 * @return boolean True if it is, false otherwise. 
	 * @param String strToken Token
	 */
	protected boolean isEllipse(String strToken) {
		return Pattern.matches("[.]+", strToken);
	}

	/**
	 * Verify if its a number.
	 * 
	 * @return true if it is, false otherwise. 
	 * @param String strToken Token
	 */
	protected boolean isNumber(String strToken) {
		//return Pattern.matches("[(]*[\"]*[(]*[A-Z]*[$]*([0-9]+[.])+", strToken);
		return Pattern.matches("[(]*[\"]*[(]*[A-Za-z]*[$]*[0-9]+[.]", strToken);
	}

	/**
	 * Determines if the token is a contraction if it is, the expanded word is returned.
	 * 
	 * @return The expanded word. (e.g., don't -> do not)
	 * @param String strToken Token
	 */
	protected Vector determineContraction(String strToken) {
		Vector vContraction = null;
		
		if(strToken.contains("'") && (vContraction = lex.lookupContraction(strToken.replace("'", "''"))) != null)
			vContraction.trimToSize();

		return vContraction;
	}
	
	/**
	 *	After tokenization, the String of sentence is 
	 *	modified if it contains a contraction. The
	 *	contraction is replaced with its corresponding
	 *	expanded word.
	 *
	 *	Example:
	 *	I don't eat ice cream.
	 *
	 *	After tokenization,
	 *	I do not eat ice cream.
	 *
	 *	@return String of the updated sentence
	 */
	public String getUpdatedSentence() {
		return strSentence;
	}

//KAT START
	public Vector tokenizeString(String strSentence, char cTarget){
	//	System.out.println("SENtENCE -->" + strSentence);
		Vector vTokens = tokenize(strSentence, cTarget);
	//	System.out.println("vTokens Tokenized -->" + vTokens);
		Vector vCompoundWords;
		int nTokenSize = vTokens.size();
		int nCompoundSize;
		TExtLexicon lex = new TExtLexicon(cTarget);
		String strTokens;
		String strCompound;
		
		for(int i = 0; i < nTokenSize; i++){
			strTokens = (String)vTokens.get(i);
			vCompoundWords = new Vector();
			vCompoundWords = lex.lookupCompound(strTokens);
			
			if(vCompoundWords == null)
				vCompoundWords = new Vector();

		//	strCompound = filterCompoundWords(strSentence, vTokens, vCompoundWords, i);
			strCompound = filterCompoundWords(vCompoundWords, vTokens, i);
			if(strCompound != null){
				nCompoundSize = (strCompound.split(" ")).length;
				for(int j = i + nCompoundSize - 1; j >= i ; j--)
					vTokens.remove(j);
				vTokens.insertElementAt(strCompound, i);
				nTokenSize = vTokens.size();
			}
		}
	//	System.out.println("vTokens Tokenized Tokenized -->" + vTokens);
		return vTokens;
	}

	public String filterCompoundWords(Vector vCompoundWords, Vector vSource, int nStart){
	//	System.out.println("vCompoundWords -->" + vCompoundWords);
		Vector vCompoundTokens;
		int nSource = vSource.size();
		int nCompoundToken;
		int nEnd;
		String strCompoundWord;
		String strCompoundSource;
		String strPrevWord = null;
		int nPrevSize = 0;
		int nWordSize;
		
	//	System.out.println("vCompoundWords -->" + vCompoundWords);
		for(Iterator i = vCompoundWords.iterator(); i.hasNext(); ){
			strCompoundWord = (String)i.next();
			nCompoundToken = (strCompoundWord.split(" ")).length;
			nEnd = nStart + nCompoundToken;
			if(nEnd <= nSource){
				strCompoundSource = "";
				for(int j = nStart; j < nEnd; j++){
					if(j > nStart)
						strCompoundSource += " ";
					strCompoundSource += (String)vSource.get(j);
				}
				nWordSize = (strCompoundWord.split(" ")).length;
				if(strCompoundSource.compareToIgnoreCase(strCompoundWord) == 0 && nWordSize > nPrevSize){
					nPrevSize = nWordSize;
					strPrevWord = strCompoundSource; 
				}
			}
		}
		System.out.println("strCompoundWord -->" + strPrevWord);
		return strPrevWord;
	}
	
/*	
	public String translatePerWord(Vector vTokens){
		TExtLexicon lex = new TExtLexicon();
		String strTarget = "";
		Vector vLookUp;
		String strTokens;
		char cFirst;
		
		for(Iterator it = vTokens.iterator(); it.hasNext(); ){
			strTokens = (String)it.next();
			vLookUp = lex.lookup(strTokens);
			
			if(vLookUp != null){
				cFirst = strTokens.charAt(0);
				strTokens = (String)vLookUp.get(0);
				if(Character.isUpperCase(cFirst)){
					cFirst = strTokens.charAt(0);
					if(strTokens.length() == 1)
						strTokens = "" + Character.toUpperCase(cFirst);
					else{
						strTokens = strTokens.substring(1);
						strTokens = "" + Character.toUpperCase(cFirst) + strTokens;
					}
				}
			}
			if(!(strTarget.equals("")) && !isPunctuation(strTokens))
				strTarget += " ";
			strTarget += strTokens;
		}
		
		if(strTarget.equals(""))
			strTarget = null;
		return strTarget;
	}*/
	
	public boolean isPunctuation(String strWord){
		Double dDoubleValue;
		
		if(strWord.equals(",") || strWord.equals("!") || strWord.equals(";") || strWord.equals("(")
			 || strWord.equals(")") || strWord.equals("{") || strWord.equals("}") || strWord.equals("[")
			  || strWord.equals("]") || strWord.equals("\"") || strWord.equals("<") || strWord.equals(">")
			   || strWord.equals("?") || strWord.equals("/") || strWord.equals(".") || strWord.equals("..."))
			   	return true;
		return false;
	}

//KAT END	
}